package AGENTS;

import java.io.*;

public class AGENTS_EMFALCON {

    ART art;

    private double[][] I;
    final int numSpace = 3;
    final int CURSTATE = 0;
    final int ACTION = 1;
    final int REWARD = 2;

    public int agentid = 0;

    public int numState;
    public int numAction;
    public int numReward = 2;

    final static int RFALCON = 0;
    final static int TDFALCON = 1;

    final int FUZZYART = 0;
    final int ART2 = 1;

    final int PERFORM = 0;
    final int LEARN = 1;
    final int INSERT = 2;

    private double[] T;
    private double[] y;
    private int[] numInput;
    private int numCode = 0;
    public double[][][] W;
    private int J;
    private boolean[] newCode;


    private int capacity = 1000;

    private double beta = (double) 1.0;
    private double epilson = (double) 0.001;
    private double gamma[] = {(double) 1.0, (double) 1.0, (double) 1.0};


    private double alpha[] = {(double) 0.1, (double) 0.1, (double) 0.1};
    private double b_rho[] = {(double) 0.2, (double) 0.2, (double) 0.5};


    public static boolean forgetting = false;
    public static boolean INTERFLAG = false;
    public static boolean detect_loop = false;
    public static boolean look_ahead = false;

    public double QAlpha = (double) 0.5;
    public double QGamma = (double) 0.1;

    public static double minQEpsilon = (double) 0.00500;
    public static double initialQ = (double) 0.5;
    public static double QEpsilonDecay = (double) 0.00050;
    public static double QEpsilon = (double) 0.50000;
    public int agent_num;


    public double this_Q;
    public double max_Q;
    public double new_Q;


    //构造
    public AGENTS_EMFALCON(int num, int numState, int numAction) throws IOException {

        art = new ART(num, capacity);

        agentid = num;

        this.numState = numState;
        this.numAction = numAction;

        numInput = new int[numSpace];   //// numSpace:0-State 1-Action 2-Reward 3-New State
        numInput[0] = numState;
        numInput[1] = numAction;
        numInput[2] = numReward;


        I = new double[numSpace][];

        for (int i = 0; i < numSpace; i++)
            I[i] = new double[numInput[i]];

        if (!rw) {
            numCode = 0;
            newCode = new boolean[capacity];
            newCode[0] = true;

            T = new double[capacity];
            y = new double[capacity];


            W = new double[capacity][][];

            for (int j = 0; j < numCode + 1; j++) {
                W[j] = new double[numSpace][];
                for (int k = 0; k < numSpace; k++) {
                    W[j][k] = new double[numInput[k]];
                    for (int i = 0; i < numInput[k]; i++)
                        W[j][k][i] = (double) 1.0;
                }
            }
        } else {
            readfile(agent_num);
        }
    }


    public void setParameters(int AVTYPE, boolean immediateReward) {

        if (AVTYPE == RFALCON) {
            QEpsilonDecay = (double) 0.00000;
            QEpsilon = (double) 0.00000;
        } else { //  QEpsilonDecay rate for TD-FALCON
            QEpsilonDecay = (double) 0.00050;
            QEpsilon = (double) 0.50000;
        }

        if (immediateReward)
            QGamma = (double) 0.5;
        else
            QGamma = (double) 0.9;
    }

    //提取环境信息
    public void setState(double[] stat) {//提取状态
        for (int i = 0; i < numState; i++) {
            I[0][i] = stat[i];
        }
    }

    public void setAction(int action) {//提取动作
        for (int i = 0; i < numInput[ACTION]; i++)
            I[ACTION][i] = 0;
        I[ACTION][action] = (double) 1.0;
    }

    public void setReward(double r) {//提取奖励
        I[REWARD][0] = r;
        I[REWARD][1] = 1 - r;
    }


    //表现部分

    //代码激活（F2节点思考T--activityF2）
    public void CodeActivition(int numSpace) {
        double top, bottom;
        for (int j = 0; j < numCode + 1; j++) {//激活函数
            for (int k = 0; k < numSpace; k++) {
                top = 0;
                bottom = (double) alpha[k];
                for (int i = 0; i < numInput[k]; i++) {
                    top += Math.min(I[k][i], W[j][k][i]);
                    bottom += W[j][k][i];
                }
                T[j] += gamma[k] * (double) (top / bottom); //Tj
            }
        }
    }


    //代码竞争（判断最好的思考）
    public int CodeCompetition() {
        double max_act = (double) -Double.MAX_VALUE;
        int J = 0;

        for (int j = 0; j < numCode + 1; j++)
            if (T[j] > max_act) {
                max_act = T[j];
                J = j;
            }
        return (J);
    }

    public boolean isNull(double[] x, int n) {
        for (int i = 0; i < n; i++)
            if (x[i] != 0) return (false);
        return (true);
    }

    //活动读出（提取记忆）
    public void WtoI(int j, int k) {

        for (int i = 0; i < numInput[k]; i++)
            I[k][i] = W[j][k][i];

    }

    int JJ;
    double alpha_art = 0.1;

    public double FALCON_PERFORM() {


        double QValue = (double) 0.0;

        CodeActivition(2); //代码激活（每个节点思考）
        J = CodeCompetition();

        for (int i = 0; i < numCode; i++) {
            y[i] = (1 - alpha_art) * y[i];
        }
        y[J] = 1;

        //从记忆中提取Q值
        art.setinputs(y);
        JJ = (int) art.ART_PERFORM(J);

        WtoI(JJ, REWARD);//F1节点提取记忆（Q值）

        if (I[REWARD][0] == I[REWARD][1] && I[REWARD][0] == 1) { //初次记忆
            QValue = (double) 0.5;
        } else
            QValue = I[REWARD][0];//有记忆就用记忆
        return (QValue);
    }


    //学习部分

    //模板匹配（判断是否合理）
    public double TemplateMatching(int k, int j) {
        double m = (double) 0.0;
        double denominator = (double) 0.0; //分母

        if (isNull(I[k], numInput[k]))
            return (1);


        for (int i = 0; i < numInput[k]; i++) {//相似度
            m = m + Math.min(I[k][i], W[j][k][i]);
            denominator = denominator + I[k][i];
        }

        if (denominator == 0)
            return (1);
        else
            return (m / denominator);
    }

    //模板学习（更新记忆）
    public void TemplateLearning(int J) {
        double rate;

        if (!newCode[J] || numCode < capacity) {
            if (newCode[J]) rate = 1;
            else rate = beta;
            for (int k = 0; k < numSpace; k++) {//再次记忆
                for (int i = 0; i < numInput[k]; i++) {
                    W[J][k][i] = (1 - rate) * W[J][k][i] + rate * Math.min(W[J][k][i], I[k][i]);
                }
            }
            if (newCode[J]) {//如果初次记忆记忆
                newCode[J] = false;
                createNewCode();//创建新的节点
            }
        }
    }

    //创建新节点    
    public void createNewCode() {

        numCode++;

        System.out.println("numCode");
        System.out.println(numCode);

        newCode[numCode] = true;
        y[numCode] = 0;

        W[numCode] = new double[numSpace][];

        for (int k = 0; k < numSpace; k++) {
            W[numCode][k] = new double[numInput[k]];
            for (int i = 0; i < numInput[k]; i++)
                W[numCode][k][i] = (double) 1.0;
        }
    }


    //完全放弃以前的记忆
    public void doOverwrite(int J) {
        for (int k = 0; k < numSpace; k++)
            for (int i = 0; i < numInput[k]; i++)
                W[J][k][i] = I[k][i];
    }

    public double FALCON_LEARN() {

        boolean reset = true,
                perfectMismatch = false;
        double QValue = (double) 0.0;
        double[] rho = new double[numSpace];
        double[] match = new double[numSpace];

        for (int k = 0; k < numSpace; k++)
            rho[k] = b_rho[k];


        CodeActivition(2); //代码激活（每个节点思考）

        //模板学习（再次记忆）
        while (reset && !perfectMismatch) {
            reset = false;
            J = CodeCompetition(); //代码竞争（判断最好的思考）
            for (int k = 0; k < numSpace; k++)
                match[k] = TemplateMatching(k, J);    //模板匹配（判断是否合理）
            if (match[CURSTATE] < rho[CURSTATE] || match[ACTION] < rho[ACTION] || match[REWARD] < rho[REWARD]) {//如果相似度过低
                if (match[CURSTATE] == 1) {//状态相似度完全相同
                    perfectMismatch = true;
                } else {//相似度低就重新初始化并更新阈值函数并重新选择
                    T[J] = -Double.MAX_VALUE;
                    reset = true;
                    for (int k = 0; k < 1; k++)
                        if (match[k] > rho[k])
                            rho[k] = Math.min(match[k] + epilson, 1);
                }
            }
        }

        art.setoutputs(J);
        art.ART_LEARN();

        if (!perfectMismatch) TemplateLearning(J);//再次记忆
        else doOverwrite(J);//再次记忆（不过完全放弃以前的记忆）
        return (QValue);
    }


    //行动策略选择行为
    public int Policy() {
        double[] qValues = new double[numAction];
        int selectedAction = -1;

        int[] validActions = new int[qValues.length];
        int maxVA = 0;

        for (int i = 0; i < numAction; i++) {
            setAction(i);
            qValues[i] = FALCON_PERFORM();   //估计当前状态下所有的行为Q值
            validActions[maxVA] = i;
            maxVA++;
        }
        //策略选择行为
        if (Math.random() < QEpsilon) {//随机行为
            int randomIndex = (int) (Math.random() * numAction);
            selectedAction = validActions[randomIndex];
            ;
        } else {//选择的行为
            double maxQ = -Double.MAX_VALUE;
            int[] doubleValues = new int[qValues.length];
            int maxDV = 0;

            for (int vAction = 0; vAction < maxVA; vAction++) {
                int action = validActions[vAction];
                if (qValues[action] > maxQ) {
                    selectedAction = action;
                    maxQ = qValues[action];
                    doubleValues[maxDV] = selectedAction;
                    maxDV = 1;
                } else if (qValues[action] == maxQ) {
                    doubleValues[maxDV] = action;
                    maxDV++;
                }
            }

            if (maxDV > 1) {   // 如果最好行为重复了，随机选一个
                int randomIndex = (int) (Math.random() * maxDV);
                selectedAction = doubleValues[randomIndex];
            }
        }
        return selectedAction;
    }

    //计算下一状态的最大Q值
    public double getMaxQValue(Boolean isdead) {
        if (isdead) {
            max_Q = 0.0;
        } else {
            for (int i = 0; i < numAction; i++) {//Q-学习
                setAction(i);
                double tmp_Q = FALCON_PERFORM();
                if (tmp_Q > max_Q) max_Q = tmp_Q;
            }
        }
        return max_Q;
    }

    //Q学习公式
    public double getNewQValue(double r) {
        double new_Q = this_Q + QAlpha * (r + QGamma * max_Q - this_Q);
        if (new_Q < 0) new_Q = 0;
        if (new_Q > 1) new_Q = 1;
        return new_Q;
    }


    public boolean rw = false;

    public void readfile(int agent_num) throws IOException {
        String path = "d:/store" + agent_num + ".txt";
        FileInputStream in = new FileInputStream(path);
        DataInputStream din = new DataInputStream(in);
        numCode = din.readInt();
        newCode = new boolean[numCode + 1];
        newCode[0] = true;

        T = new double[numCode + 1];

        W = new double[numCode + 1][][];
        for (int j = 0; j < numCode + 1; j++) {
            W[j] = new double[numSpace][];
            for (int k = 0; k < numSpace; k++) {
                W[j][k] = new double[numInput[k]];
                for (int i = 0; i < numInput[k]; i++)
                    W[j][k][i] = (double) 1.0;
            }
        }

        for (int i = 0; i < numCode; i++) {
            newCode[i] = din.readBoolean();
            for (int j = 0; j < numSpace; j++) {
                for (int k = 0; k < numInput[j]; k++) {
                    W[i][j][k] = din.readDouble();
                }
            }
        }
        din.close();
        in.close();

    }

    public void writefile(int agent_num) throws Exception {
        String path = "d:/store" + agent_num + ".txt";
        FileOutputStream out = new FileOutputStream(path);
        DataOutputStream dout = new DataOutputStream(out);

        dout.writeInt(numCode);
        for (int i = 0; i < numCode; i++) {
            dout.writeBoolean(newCode[i]);
            for (int j = 0; j < numSpace; j++) {
                for (int k = 0; k < numInput[j]; k++) {
                    dout.writeDouble(W[i][j][k]);
                }
            }
        }
        dout.close();
        out.close();
    }
}